#!/usr/bin/env python3

from get_micro_data import *
from fail_university_list import *
from config import *
from multiprocessing import Pool


'''
开始多线程获取数据
'''
def get_start(university_name):
    # micro_data = get_micro_data(university_name)
    # university_paper_data = micro_data.use_micro_interface()
    # if len(university_paper_data) == 0:
    #     print(university_name + "获取的论文数为0!!!!!!!!!!!!!!!!!!")
    # save_into_mongod(university_paper_data,university_name)
    list = [item * 1000 for item in range(162)]
    fail_university_list = []
    for offset in list:
        micro_data = get_micro_data(university_name,str(offset))
        university_paper_data = micro_data.use_micro_interface()
        if len(university_paper_data) == 0:
            fail_university_list.append({university_name:offset})
        save_into_mongod(university_paper_data,university_name)

'''
存入mongod
'''
def save_into_mongod(university_paper_data,university_name):
    for item in university_paper_data:
        db[university_name].insert(item)

'''
获取到fail_university_data的长度
'''
def get_fail_university_list(fail_university_path):


    fr = open(fail_university_path)
    fail_university_list = [item.strip().split('\t') for item in fr.readlines()]
    return fail_university_list,len(fail_university_list) - 1

'''
带入index_offset参数
'''
def add_offset(index_offset):
    pass

'''
主程序
'''
def main():
    # pool = Pool()
    # pool.map(get_start,UNIVERSITY_LIST)
    #不用进程池了

    for university_name in UNIVERSITY_LIST:
        get_start(university_name)

    # fail_university_path = "./fail_university_list.txt"
    # fail_university_list,fail_university_list_length = get_fail_university_list(fail_university_path)
    # if fail_university_list_length != 0:
    #     UNIVERSITY_LIST_FAIL = fail_university_list[:fail_university_list_length]
    #     with open (fail_university_path,'r+') as fr:
    #         fr.truncate()
    #         fr.close()
    #     print(UNIVERSITY_LIST_FAIL)
    #     # pool = Pool()
    #     # pool.map(get_start,UNIVERSITY_LIST_FAIL)
    #     fail_university_list,fail_university_list_length = get_fail_university_list(fail_university_path)
    #
    #     print(fail_university_list)
    #     print(fail_university_list_length)

if __name__ == '__main__':
    main()
